# -*- coding:utf8 -*-
import jieba
from db_manager import DBSession
import settings
from utils import help
session = DBSession(settings.db_config)
print "正在读取数据库...."
result = session.execute_query(settings.query_all_info)
print "读取完成，启动中..."


def cut_words(sentence, mode='/'):
    """
    默认使用精确模式分词
    :param sentence:
    :return:
    """
    result = ''
    result = jieba.cut(sentence)
    result = mode.join(result)
    return result


def start_fenci():
    """
    启动分词程序
    :return:
    """
    # 得到总数 判断进度
    counts = len(result)
    insert_counts = 0
    ignore_counts = 0
    for item in result:
        id = item["id"]
        if is_exist_in_db(id):
            ignore_counts = ignore_counts + 1
            if ignore_counts % 500 == 0:
                print 'find %d articles exists...' % id
            if ignore_counts > 1000:
                print "find more than %d articles, stop ...." % ignore_counts
                return
        else:
            pub_time = str(item["pub_time"])
            title = item["title"]
            title = title.replace('\'', '')
            content = item["content"]
            content = content.replace('\'', '')
            company = item["company"]
            cut_content = cut_words(title) + '/' + cut_words(content)
            cut_content.replace("\'", '')
            create_date = help.get_current_time()
            insert_sql = settings.insert_into_info_word % (str(id), pub_time, cut_content, create_date, company)
            insert_sql = insert_sql.replace('\n', '')
            try:
                session.execute_insert(insert_sql)
            except Exception as e:
                print "ERROR:", e
            insert_counts = insert_counts + 1
            if insert_counts % 500 == 0:
                print "Insert Process : %d / %d" % (insert_counts, counts - ignore_counts)


def is_exist_in_db(id):
    """
    判断article_id是否已经存在于数据库
    :param id: 文章id
    :return:
    """
    sql = settings.ignore_same_id % id
    result = session.execute_query(sql)[0]['COUNT(*)']
    if result < 1:
        return False
    else:
        return True


if __name__ == '__main__':
    start_fenci()

